#!/usr/bin/env python
# -*- coding: utf-8 -*-

import curl_async
import re
from tornado import httpclient, gen, ioloop, queues
from store_info import crawl_task
import time

async def get_task():
    """
    发送分页任务
    """
    base_url = 'http://tieba.baidu.com/mo/q---47BA1EC629DBB22F16C029A68019A549%3AFG%3D1--1-3-0--2--wapp_1439461323961_399/m?kw=%E6%B5%B7%E5%8D%97%E5%A4%A7%E5%AD%A6&lp=5009&pinf=1_2_60&sub=%E8%B7%B3%E9%A1%B5'
    for i in range(1, 9050):
        url = base_url + '&pnum=' + str(i)
        print(url)
        await get_page(url)

async def get_page(url):
    """
    获取对应url的数据
    """
    headers = {
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        'Accept-Encoding': 'gzip, deflate, sdch',
        'Accept-Language': 'en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4,zh-TW;q=0.2',
        'Connection': 'keep-alive',
        'Cookie': '',
        'Host': 'tieba.baidu.com',
        'Referer': 'http://tieba.baidu.com/mo/q---47BA1EC629DBB22F16C029A68019A549%3AFG%3D1--1-3-0--2--wapp_1439461323961_399/m?kw=%E6%B5%B7%E5%8D%97%E5%A4%A7%E5%AD%A6&lp=5011&lm=&pn=10',
        'Upgrade-Insecure-Requests': '1',
        'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.76 Safari/537.36',
    }

    response = await curl_async.fetch(url, headers=headers)

    if response and response.code == 200:
        title_list = await re_page_info(response.body)
        for info in title_list:
            topic_url, topic_title = info
            re_tid = re.compile('kz=(\d+)')
            tid_info = re_tid.findall(topic_url)
            if tid_info:
                tid = tid_info[0]
                topic_task_info = {
                    'type': 'topic',
                    'url': 'http://tieba.baidu.com/p/' + tid,
                    'title': topic_title,
                }
                await send_next_task(topic_task_info)

async def re_page_info(html):
    html = html.decode('UTF-8', 'ignore')
    re_title = re.compile('<div class="i"><a href="([^"]*)">([^<]*)</a>')
    title_list = re_title.findall(html)
    return title_list


async def send_next_task(task_info):
    """
    发送下一个任务的信息
    """
    task_db = crawl_task()
    if task_info:
        await task_db.save(task_info)

if __name__ == '__main__':
    io_loop = ioloop.IOLoop.current()
    # 使用lambda封装传递参数
    io_loop.run_sync(lambda: get_task())
